<?php
// +----------------------------------------------------------------------
// | ThinkCMF [ WE CAN DO IT MORE SIMPLE ]
// +----------------------------------------------------------------------
// | Copyright (c) 2013-2018 http://www.thinkcmf.com All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: guosheng1987@126.com 
// +----------------------------------------------------------------------
namespace plugins\crawler\service; 

use GuzzleHttp\Client;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Pool;
use GuzzleHttp\Promise;
use GuzzleHttp\Cookie\CookieJar;

use phpQuery;
//use QL\QueryList;

use plugins\crawler\model\CrawlerSiteModel;
use plugins\crawler\model\CrawlerUrlModel;
use plugins\crawler\model\CrawlerRuleModel;
use plugins\crawler\model\CrawlerCollectionModel;

use app\portal\model\PortalPostModel;
use app\portal\model\PortalCategoryModel;

use think\Db;

class CrawlerZhidaoService extends CrawlerService
{   
    //GuzzleHttp Client 
    public $client;
    //http headers
    public $headers;
    // current crawler site object
    public $site;
    // current crawler url object
    public $url;

    public $collections;

    public $config = [
        'client' => [
            // Base URI is used with relative requests
            'base_uri' => 'https://zhidao.baidu.com',
            // default options.
            'timeout'  => 30.0,
            'verify'   => false,
            'delay'    => 3000,
        ],
        'headers' => [
            'User-Agent'      => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
            'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Encoding' => 'gzip, deflate',
            'Accept-Language' => 'zh-CN,zh;q=0.9',
            'Cache-Control'   => 'max-age=0',
            'Connection'      => 'keep-alive',
            'Host'            => 'zhidao.baidu.com',
            'Cookie'          => '',
        ]
    ];
    /**
     * 构造函数
     *
     */
    public function __construct() {}   

    
    /**
     * 设置请求头信息
     * @access public
     * @return void
     */
    public function setCookie($headers)
    {
        $cookie = [
            'Hm_lvt_9c9e4e5987185e705fdd349134ab467b='. time(),
            'Hm_lpvt_9c9e4e5987185e705fdd349134ab467b='. time(),
            'BAIDUID=63779F8FB114113CEA5F98F4A3A82C7A:FG=1',
            'PSTM='.time(),
            'BIDUPSID=2CA27BF4F42B99FFC36826B25C18FA1B',
            'BDORZ=B490B5EBF6F3CD402E515D22BCDA1598',
            'H_PS_PSSID=1427_26459_21113_22159',
            'IKUT=4480',
            'PSINO=3',
            'BDRCVFR[C0p6oIjvx-c]=rJZwba6_rOCfAF9pywd',
        ];

        $headers['Cookie'] = implode('; ', $cookie);

        return $headers;
    }

    /**
     * get collection path and Thumb img from list page
     * @access public
     * @param  String $contents
     * @return Array $collection
     */
    public function parseCollectionPathByContents($contents)
    {
        $collections = [];

        if($contents)
        {
            phpQuery::newDocumentHTML($contents);
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            foreach(pq('div.list-inner dl dt a') as $a)
            {
                $path  = pq($a)->attr('href');
                
                $parseInfo = parse_url($path);
                
                $item['paths'] = $parseInfo['path'].'?ie=gbk';

                $collections[] = $item;
            }

            phpQuery::$documents = null;
        
            //$collectionPaths = $qlData->all();
        }

    
        return $collections;
    }

    /**
     * parse question title from article page html
     * @access public
     * @param  String $contents
     * @return String $title
     */
    public function parseCollectionTitleByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents, 'utf-8');
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            $title = pq('div#wgt-ask h1 span.ask-title')->text();

            phpQuery::$documents = null;
        }

        return $title;
    }

    /**
     * parse question explanation from article page html
     * @access public
     * @param  String $contents
     * @return String $explanation
     */
    public function parseCollectionExplanationByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents, 'utf-8');
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            $explanation = pq('div#wgt-ask div span.con')->html();

            phpQuery::$documents = null;
        }

        return $explanation;
    }

    /**
     * parse question answer from article page html
     * @access public
     * @param  String $contents
     * @return String $answer
     */
    public function parseCollectionAnswerByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents, 'utf-8');

            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            $answer = '';
            
            if(pq('div')->hasClass('wgt-best') || pq('div')->hasClass('wgt-answers'))
            {
                foreach(pq('div.line.content') as $ansDiv)
                {
                    $acontent = pq($ansDiv)->find('[accuse=aContent]')->html();

                    $answer .= '<div class="row" style="padding:15px;">'.$acontent.'</div>';
                }
            }
            phpQuery::$documents = null;

        }

        return $answer;
    }

    /**
     * parse related questions from article page html
     * @access public
     * @param  String $contents
     * @return String $related
     */
    public function parseCollectionRelatedQuestionsByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents, 'utf-8');

            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            $related = '';

            if(pq('div')->hasClass('related-list'))
            {
                foreach(pq('div.related-list.line li') as $li)
                {
                    $quesText = pq($li)->find('span.related-restrict-title.grid')->html();
                    $quesLink = pq($li)->find('a.related-link.related-link-zd')->attr('href');
                    $related .= '<p class="text-primary"><a href="'.$this->site->siteurl .$quesLink. '" target="_blank">'.$quesText.'</a></p>';
                }
            }

            phpQuery::$documents = null;

        }

        return $related;
    }
    
    
    /**
     * generate rand accounts
     * @access public
     * @return Array $accounts
     */
    public function getRandAccounts()
    {
        $maxid = Db::name('accounts')->max('id');
        $times = rand(3,8);
        $ids = [];
        
        while(true)
        {
            $id = rand(1, $maxid);
            if( !in_array($id, $ids) ) {
                $ids[] = $id;
            }
            if(count($ids) == $times) break;
        }
        
        $accounts = Db::name('accounts')
            ->where(['id' => ['in', $ids]])
            ->select();

        return $accounts;
    }

   
    /**
     * generate portal data
     * @access private
     * @param  String $categoryName
     * @param  Array $collection 
     * @return Array data
     */
    private function makePortalData($categoryName, $collection)
    {
        //社工库撸账号

        $data = [];
        
        $data['title'] = $categoryName.'账号'.date('Y').'年'.date('m').'月'.date('d').'日更新  ';
        
        $tails = Db::name('dictionary')
            ->where(['id' => $this->url->crlrule->dictionary_id])
            ->value('tailwords');

        //$tails = $this->url->crlrule->tails;

        $tailsArr = explode(',', $tails);

        $randKeys = !empty($tailsArr)? array_rand($tailsArr, 1): [];
        
        if(!empty($randKeys))
        {
            $data['title'] .= $tailsArr[$randKeys];
        }

        // make
        $accounts = $this->getRandAccounts();

        $accountHtml = '';
        foreach($accounts as $account)
        {
            $accountHtml .= '<p>账号：' . $account['email'] . ' 密码：'. $account['password'].'</p>';
        }

        $data['excerpt'] = mb_substr(strip_tags($data['title'].'  '.$accountHtml), 0, 150);

        $bootomHtml = '分享喵(www.fenxiangmiao.com)，将持续不断的为网友们分享'.$categoryName.'账号资源，请分享给你们的朋友们哦。免费账号由于使用人数过多，5分钟内可能失效，如需稳定会员加微信<b>machen77</b>，官网半价秒发货<br>';
        
        $qrimageHtml = '<img src="/themes/'.cmf_get_current_theme().'/public/assets/images/qrcode.jpg" style="box-sizing: border-box; border: 0px; vertical-align: middle; max-width: 100%; height: auto !important;"><br>';
        
        $content = $data['title']. $accountHtml .$bootomHtml. $qrimageHtml . '<br/>';
        $content = $content.'<hr><h4>'.$collection['title'].'</h4><p><em>'.$collection['explanation'].'</em></p>'.$collection['content'].'';
        
        if($collection['related'])
        {
            $content = $content . '<hr><h4>关联问题</h4>' . $collection['related'];
        }
        
        $data['content'] = $content;

        return $data;
    } 

    
    /**
     * save collection to db
     * @access public
     * @param  Int $index 
     * @return void
     */
    public function saveCrawlerData($index)
    {
        //先删除关联表，防止重复
        $this->deletePortal($this->collections[$index]['collection_id']);
        
        $categoryId = $this->url->crlrule->category_id;
        
        $categoryName = $this->getPortalCategoryNameById($categoryId);

        $portal = $this->makePortalData($categoryName, $this->collections[$index]);
        
        $postId = $this->addPortalAndTag($categoryId, $portal['title'], $portal['excerpt'] ,$portal['content']);
        
        $this->addCrawlerCollection($postId, $this->collections[$index]);
    }
    
    /**
     * request list url and aget 
     * @access public
     * @return string
     */
    public function reuqestListPage()
    {
        $client = new Client($this->config['client']);
        
        $headers = $this->config['headers'];

        $headers = $this->setCookie($headers);

        $response = $client->request('GET', $this->url->listurl, [
            'allow_redirects' => false,
            'headers'         => $headers,
        ]);
        
        $contents = $response->getBody()->getContents();

        return $contents;
    }

    
    /**
     * request Qrcode image url in article page
     * @access public
     * @return void
     */
    public function requestCollectionPages()
    {
        $client = new Client($this->config['client']);

        $headers = $this->config['headers'];

        $headers = $this->setCookie($headers);

        $requests = function ($headers) {
            foreach ($this->collections as $i => $collection) {
                yield new Request('GET', $this->site->siteurl . $collection['paths'], $headers);                                
            }
        };

        $pool = new Pool($client, $requests($headers), [
            'concurrency' => 3,
            'fulfilled'   => function ($response, $index) {
                
                $contents = $response->getBody()->getContents();

                $matchtimes = preg_match('/\d+/', $this->collections[$index]['paths'], $matches);
                
                $this->collections[$index]['collection_id']   = $matches[0];

                $this->output->writeln("page url ".$this->collections[$index]['paths'] .' start!'); 

                $title = $this->parseCollectionTitleByContents($contents);
                
                $explanation = $this->parseCollectionExplanationByContents($contents);
                
                $answer = $this->parseCollectionAnswerByContents($contents);

                $related = $this->parseCollectionRelatedQuestionsByContents($contents);

                $this->collections[$index]['title']           = $title;
                $this->collections[$index]['explanation']     = $explanation;
                $this->collections[$index]['content']         = $answer;
                $this->collections[$index]['related']         = $related;

                $this->saveCrawlerData($index);
            
                $this->output->writeln("page url ".$this->collections[$index]['paths'] .' collect data success!');         
            },
            'rejected'   => function ($reason, $index) {
                
                //var_dump($reason);

                // this is delivered each failed request
                $this->output->writeln("page url ".$this->collections[$index]['paths'] .' request fail!');

            },
        ]);

        // Initiate the transfers and create a promise
        $promise = $pool->promise();

        // Force the pool of requests to complete.
        $promise->wait();
        
    }
    
    /**
     * get Collection ids not in table
     * @access public
     * @return void
     */
    public function getNoSavedCollectionIds()
    {
        $currentIds = $savedIds = [];
        foreach($this->collections as $i => $collection)
        {
            $matchtimes = preg_match('/\d+/', $collection['paths'], $matches);
            if($matches && $matches[0])
            {
                $currentIds[$i] = (string) $matches[0];
            }
        }

        $savedIds = Db::name('crawler_collection')
            ->where(['crawler_site_id' => $this->site->id])
            ->where(['collection_id' => ['in', $currentIds]])
            ->column('collection_id');

        foreach($currentIds as $i => $collection_id)
        {
            if( in_array($collection_id, $savedIds) )
            {
                unset($this->collections[$i]);
            }
        }
        
        $this->collections = array_values($this->collections);

    }

    /**
     * start crawler job
     * 
     * @return array
     */
    public function start()
    {
        $this->site = CrawlerSiteModel::find(3);

        $urls = CrawlerUrlModel::where('crawler_site_id', $this->site->id)->select();
        
        foreach($urls as $url)
        {   
            $this->url = $url;
           
            $contents = $this->reuqestListPage();

            $this->output->writeln("list url: ". $this->url->listurl.' request success!');
            
            $this->collections = $this->parseCollectionPathByContents($contents);
            
            /*
            this is for test special page 
            $collections = $this->collections;
            $tmpCol = [];
            foreach($collections as $i => $collection)
            {
                $matchtimes = preg_match('/\d+/', $collection['paths'], $matches);

                if($matches[0] == '460951961345051205')
                {
                    $tmpCol[] = $collection;
                }
            }

            $this->collections = $tmpCol;
            */
            
            $this->getNoSavedCollectionIds();
            
            if(count($this->collections))
            {
                $this->collections = array_slice($this->collections, 0, 2);
                
                $this->requestCollectionPages();
                
                $this->output->writeln("list url ".$this->url->listurl.' collected success!');
            }
            else
            {
                $this->output->writeln("list url ".$this->url->listurl.' already done!');
            }
        }
        $this->output->writeln("Command executed success!");
        
    }
    
}